#!/usr/bin/perl -w
use strict;
use warnings;
use Encode;

my @queries = qw(电脑 网络 科技 商务 商贸 个体 广告 文化 发展 经销);

my %site_map = ('北京_ganji' => 'bj.ganji.com',
	'天津_ganji' => 'tj.ganji.com',
	'北京_58' => 'bj.58.com',
	'天津_58' => 'tj.58.com'
	);

my %site_map_ganji;
my %site_map_58;
my %city_map_koubei;
my %city_map_huicong;

sub get_site {
	my ($city, $site) = @_;
	return 'bendi.koubei.com' if ($site eq 'koubei');
	return 'search.china.alibaba.com' if ($site eq 'alibaba');
	return 'www.jc.net.cn' if ($site eq 'jiancai');
	return 's.hc360.com' if ($site eq 'huicong');
	return 'www.86mai.com' if ($site eq 'maiwang');
	return $site_map_ganji{$city}.'.ganji.com' if ($site eq 'ganji');
	if ($site eq '58') {
		return '' if (!defined($site_map_58{$city}));
		return $site_map_58{$city}.'.58.com' if ($site eq '58');
	}
	return $site_map{$city.'_'.$site};
}

my %path_map = (
	'电脑_ganji' => 'site/s/_',
	'网络_ganji' => 'site/s/_',
	'科技_ganji' => 'site/s/_',
);

my %jiancai_city_map = (
	'北京' => 1,
	'天津' => 2
);

my %huicong_city_map = (
	'北京' => '中国:北京',
	'天津' => '中国:天津',
);

my %maiwang_city_map = (
	'北京' => 1,
	'天津' => 3
);

sub get_path {
	my ($query, $site, $city) = @_;
	if ($site eq 'ganji') {
		return 'site/s/_';
	}
	if ($site eq '58') {
		if ($query eq '电脑') {
			return 'diannao/key_';
		}
		elsif ($query eq '广告') {
			return 'zpguanggao/?key=%25u5e7f%25u544a&cmcskey=&final=1&jump=1&nouse=';
		}
		elsif ($query eq '经销') {
			return 'cheliangfuwu/?key=%25u7ecf%25u9500&cmcskey=%25u7ecf%25u9500&final=1&jump=1&nouse=';
		}
		else {
			return 'sou/jh_';
		}
	}
	if ($site eq 'koubei') {
		my %city_map = (
			'北京' => 99,
			'天津' => 2260
		);
		return '' if (!defined($city_map_koubei{$city}));
		return "search/searchstore.html?sourcekey=&position=0&q=&sf=1&ei=utf8&city=$city_map_koubei{$city}&keyword=";
	}
	if ($site eq 'alibaba') {
		return 'selloffer/k-'.url_encode(utf8_to_gb2312($query)).'_s-img_city-'.url_encode(utf8_to_gb2312($city)).'_filt-y_lessThanQuantityBegin-true_pageSize-40_n-y.html?nouse=';
	}
	if ($site eq 'jiancai') {
		return 'jc_usm2/usm_changshang.asp?Area='.$jiancai_city_map{$city}.'&Key='.utf8_to_gb2312($query).'&nouse=';
	}
	if ($site eq 'huicong') {
		return '' if (!defined($city_map_huicong{$city}));
		return 'supply/'.url_encode(utf8_to_gb2312($query)).'.html?L=10&z='.url_encode(utf8_to_gb2312($city_map_huicong{$city})).'&nouse=';
	}
	if ($site eq 'maiwang') {
		return 'company/search.php?kw='.url_encode($query).'&vip=0&type=0&catid=0&mode=0&areaid='.$maiwang_city_map{$city}.'&size=0&mincapital=&maxcapital=&x=63&y=15&nouse=';
	}
}

sub get_template {
	my ($query, $city, $site) = @_;
	if ($site eq '58') {
#		return '<td class="t"[\s\w\=\"]*>\s*<a href="([\w\/\:\.]+)" target="_blank" class="t"[\sa-z\=\"]*>\s*([\d\D]+?)</a>';
		return '<td class="t"[\s\w\=\"]*>\s*<a href="([\w\/\:\.]+)" [\sa-z\=\"_]*?>\s*([\d\D]+?)</a>';
	}
	if ($site eq 'ganji') {
		if ($query eq '电脑') {
			return '<\/i>[\s]+<a href="([\/\w\._]+)" target="_blank">([\d\D]+?)</a>';
		}
		elsif ($query eq '网络' || $query eq '科技' || $query eq '商务' || $query eq '商贸' || $query eq '个体' || $query eq '广告' || $query eq '文化' || $query eq '发展' || $query eq '经销') {
#			return '<dt><a href="[\s]*([\/\w\._]+)[\s]*" target="_blank" class="list_title">[\s]*([\d\D]+?)<\/a>';
			return '                <a href="([\w\/_\.]+)" target="_blank">([\d\D]+?)</a>'
		}
	}
	if ($site eq 'koubei') {
		return '<h2 class="store-name"><a href="http:\/\/ju.atpanel.com/\?url=([\w:\.\/\?\=\-\&;]+?)" target="_blank">([\d\D]+?)</a></h2>';
	}
	if ($site eq 'alibaba') {
		return '<h3 class="summary">\s+[\d\D]+? href="([\w:\/\.]+?)" target="_blank" title="([\d\D]+?)"';
	}
	if ($site eq 'jiancai') {
		return '<td width="444" height="28" align="left" class=\'[\w]+\'><a href="([\w\/\-\.]+)" target="_blank" >([\d\D]+?)<\/a><\/td>';
	}
	if ($site eq 'huicong') {
		return 'href="([\w:\/\.]+)" target="_blank" style="font-size:14px; font-weight:bold; text-decoration:underline;">([\d\D]+?)<\/a>';
	}
	if ($site eq 'maiwang') {
		return '<a href="([\w:\/\.]+)" target="_blank"><strong class="px14">([\d\D]+?)<\/strong><\/a>';
	}
}

sub decode_title {
	my ($title, $domain) = @_;
	if ($domain eq 'koubei' || $domain eq 'alibaba' || $domain eq 'jiancai' || $domain eq 'huicong') {
		return encode('UTF8', decode('gb2312', $title));
	}
	return $title;
}

sub get_result_url {
	my ($site, $path) = @_;
	if (index($path, 'http://') == 0) {
		return $path;
	}
	if (index($path, '/') == 0) {
		return 'http://'.$site.$path;
	}
}

sub extract_details {
	my ($domain, $query, $url) = @_;
	my $content = `wget $url -q -O -`;
#print $url."\n";
#print $content;
#exit();
	my $file_name = '';
	my $phone_num = '';
	my $contact_person = '';
	my $phone_digest = '';
	if ($domain eq '58') {
#		if ($content =~ /联系电话：<\/i>[\s]*<span class="[\w\s]*?tel"><img src='([\d\D]+?)'/) {
#		if ($content =~ /联系[\d\D]+?：<\/[\w]+>[\s]*<span [\w\s="_]+?><img src='([\d\D]+?)'/) {
		if ($content =~ /联系[\d\D]+?<img src='([\d\D]+?)'/) {
			if (index($1, 'http') >= 0) {
				$file_name = `echo "$1" | md5sum`;
				$file_name = substr($file_name, 0, 32);
				my $cmd = "wget \"$1\" -q -O images/$domain/$file_name";
				`$cmd`;
				if ($1 =~ /v=(\w+)/) {
					$phone_digest = substr($1, 0, 16);
				}
#print "$file_name\n";
			}
			else {
				print "wrong1\n";
			}
		}
		elsif ($content =~ /联系[\d\D]+：<\/[\w]+>\s*<span [\w\s="]+?>([\d\-]{7,})<\/span>/) {
			$phone_num = $1;
		}
		elsif ($content =~ /id="t_phone">([\d\-]{7,})<\/span>/) {
			$phone_num = $1;
		}
		elsif ($content =~ /<img src='(http:\/\/image\.58\.com\/showphone\.aspx\?t=[\D\d]+)'/) {
			print "phone image url $1\n";
		}
		if ($content =~ /linkman:'([\d\D]*?)'/) {
			$contact_person = $1;
		}
	}
	return ($file_name, $phone_num, $contact_person, $phone_digest);
}

sub url_encode {
	if (scalar @_ != 1) {
		print STDERR "usage: ./$0 plaintext\n";
		return '';
	}
	my $ret = $_[0];
	$ret =~ s/(\W)/sprintf("%%%02X", unpack("C", $1))/eg;
	return $ret
}

sub utf8_to_gb2312 {
	return encode('gbk', decode('UTF8', $_[0]));
}

sub load_map {
	my %ret;
	open IN, $_[0] or die("fail	open $_[0]\n");
	while (my $line = <IN>) {
		chop $line;
		my @arr = split("\t", $line);
		next if (scalar @arr != 2);
		$ret{$arr[0]} = $arr[1];
	}
	close IN;
	return %ret;
}

sub init() {
	%site_map_ganji = load_map('city_map_ganji');
	%site_map_58 = load_map('city_map_58');
	%city_map_koubei = load_map('city_map_koubei');
	%city_map_huicong = load_map('city_map_huicong');
}

sub load_all_cities {
	my $city_str = `cut -f 1 city_map_* | grep -v 北京 | grep -v 上海 | sort -u`;
	my @cities = split("\n", $city_str);
	my @result;
	foreach my $city (@cities) {
		push @result, $city if (length($city) >= 2);
	}
	return @result;
}

sub load_every_cities {
	my $city_str = `cut -f 1 city_map_* | sort -u`;
	my @cities = split("\n", $city_str);
	my @result;
	foreach my $city (@cities) {
		push @result, $city if (length($city) >= 2);
	}
	return @result;
}

sub load_city_digit_map {
	my %city_map;
	open IN, 'city_digit_map.'.$_[0] or die("fail\topen city_digit_map.$_[0]\n");
	while (<IN>) {
		chop $_;
		my @arr = split("\t");
		next if (@arr != 2);
		next if (defined($city_map{$arr[0]}));
		$city_map{$arr[0]} = $arr[1];
	}
	close IN;
	return %city_map;
}

sub execute_scalar {
        my ($sql, $conn) = @_;
        my $request = $conn->prepare($sql);
        $request->execute();
        my ($result) = $request->fetchrow_array;
	$result = '' if (!defined($result));
        return $result;
}

my @proxies;
my $proxy_index = 0;
sub load_proxies {
#	my %proxies = load_map('proxy');
#	@proxies = keys %proxies;
	open IN, 'proxy';
	while (<IN>) {
		push @proxies, $1 if (/([\S]+)/);
	}
	close IN;
	print STDERR "".(scalar @proxies)." proxies loaded\n";
	return @proxies;
}

sub fetch_url_via_proxy {
	my $failure_count = 0;
	while (1) {
#		print "curl -s -i -x $proxies[$proxy_index] -m 5 '$_[0]'\n";
		$proxy_index = rand(@proxies);
		my $proxy = $proxies[$proxy_index];
		my $content = `curl -A 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)' -s -i --speed-time 5 --speed-limit 50000 --connect-timeout 60 -x $proxy -m 300 '$_[0]'`;
#		my $content = `curl -s -i -x $proxies[$proxy_index] -m 5 '$_[0]'`;
#		my $content = `curl -i -x $proxies[$proxy_index] '$_[0]'`;
#		$proxy_index = ($proxy_index + 1) % @proxies;
		my $return_code = 0;
		my $content_length = length($content);
		$return_code = substr($content, 9, 3) if ($content_length >= 2048);
		if ($content_length < 2048 || $return_code ne '200') {
#			print "delete proxy $proxies[$proxy_index]\n";
			for (my $i = 0; $i < @proxies; ++$i) {
			 	if ($proxies[$i] eq $proxy) {
					$proxies[$i] = pop @proxies;
					last;
				}
			}
			if (@proxies < 50) {
				load_proxies();
#				print "reload proxy\n";
			}
			if (++$failure_count > 20) {
#				print "give up. ".(scalar @proxies)." proxies left\n";
				return '';
			}
		}
		else {
			return $content;
		}
	}
}

sub url_decode {
	my $ret = $_[0];
	$ret =~ s/%([a-fA-F0-9][a-fA-F0-9])/pack("C", hex($1))/eg;
	$ret =~ s/\+/ /g;
	return $ret; 
}


sub get_date_str {
	my $num_diff = $_[0];
	my $str_diff = '';
	if ($num_diff >= 0) {
		$str_diff = "$num_diff days";
	}
	else {
		$num_diff = -$num_diff;
		$str_diff = "$num_diff days ago";
	}
	my $date_str = `date -d "$str_diff" +%Y-%m-%d`;
	chop $date_str;
	return $date_str;
}

sub gen_random_ip {
	my @slices;
	for my $i (0...3) {
		$slices[$i] = int(rand(255));
	}
	return join('.', @slices);
}
#print gen_random_ip()."\n";

1;
